import pickle
import hashlib
from redis import Redis
from Spider.util.MyFuncClass import HttpRequest
from Spider.settings import REDIS_CONFIG, FAIL_TIME
from Spider.Exception.exceptions import ObjectTypeError


class UrlFilter(object):
    SUCCESS_LIST = []
    FAIL_LIST = []

    def process_url(self, obj):
        md5 = self.get_md5(obj.url)
        if md5 not in UrlFilter.SUCCESS_LIST:
            UrlFilter.SUCCESS_LIST.append(md5)
            return obj
        else:
            obj.fail_time += 1
            if obj.fail_time > FAIL_TIME:
                UrlFilter.SUCCESS_LIST.remove(md5)
                UrlFilter.FAIL_LIST.append(md5)
                return False
            return obj

    def get_md5(self, url):
        if isinstance(url, str):
            url = url.encode('utf-8')
        m = hashlib.md5()
        m.update(url)
        return m.hexdigest()

    def get_number(self):
        return {'success_number': len(UrlFilter.SUCCESS_LIST),
                'fail_number': len(UrlFilter.FAIL_LIST),
                'crawl_number': len(UrlFilter.SUCCESS_LIST) + len(UrlFilter.FAIL_LIST)}


class FifoQueue(object):
    __LIST_NAME = REDIS_CONFIG.get('url_list_name')

    def __init__(self):
        self.redis = Redis(host=REDIS_CONFIG.get('host'), port=REDIS_CONFIG.get('port'), db=0)
        self.filter = UrlFilter()

    def push(self, obj):
        if isinstance(obj, HttpRequest):
            obj = self.filter.process_url(obj)
            if obj:
                self.redis.lpush(FifoQueue.__LIST_NAME, pickle.dumps(obj))
        else:
            raise ObjectTypeError('Current obj is not HttpRequest Object')

    def pop(self):
        # 元组    0:表名   1:字节类型数据
        tup = self.redis.brpop(FifoQueue.__LIST_NAME, timeout=20)
        if isinstance(tup, tuple):
            return pickle.loads(tup[1])
        return None


